import requests
from lxml import etree
import pymongo
import re

mongo_conn = pymongo.MongoClient()
counter = 0

for page in range(1, 136 + 1):
    # 模拟请求 获取包含数据的html字符串
    url = f"https://book.zongheng.com/store/c0/c0/b0/u4/p{page}/v0/s9/t0/u0/i1/ALL.html"
    response = requests.get(url)
    html_str = response.content.decode()
    # 使用xpath路径表达式进行数据解析  针对html字符串进行初始化 构造dom树
    root = etree.HTML(html_str)
    # 按行按条获取数据 先获取包含所有需要字段的父节点
    book_list = root.xpath("//div[@class='bookinfo']")
    for book in book_list:
        counter += 1
        # . 从当前节点继续往后查询节点 采集数据
        book_name = book.xpath(".//div[@class='bookname']/a/text()")[0]
        book_url = book.xpath(".//div[@class='bookname']/a/@href")[0]
        author, category = book.xpath(".//div[@class='bookilnk']/a/text()")
        state, col_num = book.xpath(".//div[@class='bookilnk']/span/text()")
        # 数据清洗（数据去重 去除无用符号）
        state = re.sub(r"\s", "", state)
        col_num = re.sub(r"[\s总收藏]", "", col_num)
        book_intro = book.xpath(".//div[@class='bookintro']/text()")[0]
        book_update = book.xpath(".//div[@class='bookupdate']/a/text()")[0]
        print(page, counter, book_name, book_url, author, category, state, col_num, book_update)
        mongo_conn.zongheng.book.insert_one({
            "book_name": book_name,
            "book_url": book_url,
            "author": author,
            "category": category,
            "state": state,
            "col_num": col_num,
            "book_intro": book_intro,
            "book_update": book_update
        })
